#coding=utf-8

#本函式庫用來處理html原始碼，將之轉換為各種需要的形式

import html.parser, sys, os.path

from PyQt4.QtGui import *
from PyQt4.QtCore import *

class totxtParser(html.parser.HTMLParser):
    '''將feed入的html標籤去掉，儘可能轉為txt格式'''
    def __init__(self, strict = True):
        if sys.version_info[1] >= 2:
            super().__init__(strict)
        else:
            super().__init__()
        self.reset()
    def reset(self):
        super().reset()
        self.script = 0
        self.inbody = False
        self.instyle = False
        self.text = ''
    def handle_starttag(self, tag, attrs):
        if tag == 'script' or tag == 'javascript' or tag == 'iframe':
            self.script += 1
        elif tag == 'body':
            self.inbody = True
        elif tag == 'style':
            self.instyle = True
    def handle_endtag(self, tag):
        if tag == 'script' or tag == 'javascript' or tag == 'iframe':
            self.script -= 1
            if self.script < 0:
                self.script = 0
        elif tag == 'body':
            self.inbody = False
        elif tag == 'style':
            self.instyle = False
    def handle_data(self, data):
        if self.inbody == True and self.script == 0 and self.instyle == False:
            self.text += data
    def pop(self):
        '''取回資料並重設'''
        text = self.text
        self.reset()
        return text

def totxt(htmlstr):
    '''輸入html，儘可能傳回純文字'''
    try:
        p = totxtParser()
        p.feed(htmlstr)
        return p.pop()
    except html.parser.HTMLParseError:
        if sys.version_info[1] >= 2:
            p = totxtParser(False)
            p.feed(htmlstr)
            return p.pop()
        else:
            return ''

class headingParser(html.parser.HTMLParser):
    '''將feed入的html中，title的標題匯出來，轉成QTreeWidgetItem的列表'''
    def __init__(self, strict = True):
        if sys.version_info[1] >= 2:
            super().__init__(strict)
        else:
            super().__init__()
        self.reset()
    def reset(self):
        super().reset()
        self.data = []
        self.index = {} #某特定title出現的位置
        self.inheading = []
        for i in range(6):
            self.inheading.append(False)
    def handle_starttag(self, tag, attrs):
        if tag == 'h1':
            self.inheading[0] = True
        elif tag == 'h2':
            self.inheading[1] = True
        elif tag == 'h3':
            self.inheading[2] = True
        elif tag == 'h4':
            self.inheading[3] = True
        elif tag == 'h5':
            self.inheading[4] = True
        elif tag == 'h6':
            self.inheading[5] = True
    def handle_endtag(self, tag):
        if tag == 'h1':
            self.inheading[0] = False
        elif tag == 'h2':
            self.inheading[1] = False
        elif tag == 'h3':
            self.inheading[2] = False
        elif tag == 'h4':
            self.inheading[3] = False
        elif tag == 'h5':
            self.inheading[4] = False
        elif tag == 'h6':
            self.inheading[5] = False
    def handle_data(self, data):
        for i in range(6):
            if self.inheading[i]:
                try:
                    self.index[data] += 1
                except KeyError:
                    self.index[data] = 0
                #newItem = QTreeWidgetItem()
                #newItem.setText(0, data) #資料
                #newItem.setText(1, ' ' + '▶'*i + ' ' + data) #顯示
                #newItem.setText(2, str(self.index[data])) #出現於第幾次
                #newItem.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
                #self.data.append(newItem)
                #資料意思：(標題文字, 第幾級(0起算) ,相同文字第幾次出現（零起算）)
                self.data.append((data, i, self.index[data]))
                break
    def pop(self):
        '''取回資料並重設'''
        data = self.data
        self.reset()
        return data

def getHeadingItems(htmlstr):
    '''輸入html，傳回heading列表'''
    try:
        p = headingParser()
        p.feed(htmlstr)
        return p.pop()
    except html.parser.HTMLParseError:
        if sys.version_info[1] >= 2:
            p = headingParser(False)
            p.feed(htmlstr)
            return p.pop()
        else:
            return ''

class imgSrcParser(html.parser.HTMLParser):
    '''找出所有圖檔的src為置並回傳'''
    def __init__(self, strict = True):
        if sys.version_info[1] >= 2:
            super().__init__(strict)
        else:
            super().__init__()
        self.reset()
    def reset(self):
        super().reset()
        self.data = set()
    def handle_starttag(self, tag, attrs):
        if tag == 'img':
            for key,value in attrs:
                if key == 'src':
                    #強制使用正斜線
                    self.data.add(os.path.basename(value))
                    break
    def handle_startendtag(self, tag, attrs):
        self.handle_starttag(tag, attrs)
    def pop(self):
        '''取回資料並重設'''
        data = self.data
        self.reset()
        return data

def getImgSrc(htmlstr):
    '''輸入html，傳回圖片路徑列表'''
    try:
        p = imgSrcParser()
        p.feed(htmlstr)
        return (True ,p.pop())
    except html.parser.HTMLParseError:
        #這是用來做回收的，因此不允許不完整的解析動作，不完整視為失敗
        return (False ,None)

import re

def html_to_text(data):
    # remove the newlines
    data = data.replace("\n", " ")
    data = data.replace("\r", " ")
    # replace consecutive spaces into a single one
    data = " ".join(data.split())
    # get only the body content
    bodyPat = re.compile(r'< body[^<>]*?>(.*?)< / body >', re.I)
    result = re.findall(bodyPat, data)
    data = result[0]
    # now remove the java script
    p = re.compile(r'< script[^<>]*?>.*?< / script >')
    data = p.sub('', data)
    # remove the css styles
    p = re.compile(r'< style[^<>]*?>.*?< / style >')
    data = p.sub('', data)
    # remove html comments
    # p = re.compile(r'')
    # data = p.sub('', data)
    # remove all the tags
    p = re.compile(r'<[^<]*?>')
    data = p.sub('', data)
    return data
